diff --git a/CA.nuspec b/CA.nuspec index a8a92c66..69def6de 100644 --- a/CA.nuspec +++ b/CA.nuspec @@ -4,7 +4,7 @@ continuous-clearing - 5.0.0 + 5.1.0 Siemens AG continuous-clearing contributors https://github.com/siemens/continuous-clearing @@ -13,7 +13,7 @@ false The License clearing tool helps the Project Manager/Developer, to reduce the manual effort and enable the faster license clearing process, - by automatically identifying the third party oss components used in their project(i.e., npm, nuget, maven,python and Debian type) and it creates them in the sw360 and fossology + by automatically identifying the third party oss components used in their project(i.e., npm, nuget, maven, python, conan and Debian type) and it creates them in the sw360 and fossology for clearing license diff --git a/doc/UsageDoc/CA_UsageDocument.md b/doc/UsageDoc/CA_UsageDocument.md index e5b040da..f2b04fab 100644 --- a/doc/UsageDoc/CA_UsageDocument.md +++ b/doc/UsageDoc/CA_UsageDocument.md @@ -46,20 +46,21 @@ # Introduction -The Continuous Clearing Tool helps the Project Manager/Developer to automate the sw360 clearing process of 3rd party components. This tool scans and identifies the third-party components used in a NPM, NUGET, MAVEN,PYTHON and Debian projects and makes an entry in SW360, if it is not present. Continuous Clearing Tool links the components to the respective project and creates job for code scan in FOSSology.The output is an SBOM file which has a nested description of software artifact components and metadata. +The Continuous Clearing Tool helps the Project Manager/Developer to automate the sw360 clearing process of 3rd party components. This tool scans and identifies the third-party components used in a NPM, NUGET, MAVEN, PYTHON, CONAN and Debian projects and makes an entry in SW360, if it is not present. Continuous Clearing Tool links the components to the respective project and creates job for code scan in FOSSology.The output is an SBOM file which has a nested description of software artifact components and metadata. Continuous Clearing Tool reduces the effort in creating components in SW360 and identifying the matching source codes from the public repository. Tool eliminates the manual error while creating component and identifying correct version of source code from public repository. Continuous Clearing Tool harmonize the creation of 3P components in SW360 by filling necessary information. # Continuous Clearing Tool workflow diagram - Package Identifier - - [NPM/NUGET/MAVEN/PYTHON](../usagedocimg/packageIdentifiernpmnuget.PNG) + + - [NPM/NUGET/MAVEN/PYTHON/CONAN](../usagedocimg/packageIdentifiernpmnuget.PNG) - [Debian](../usagedocimg/packageIdentifierdebian.PNG) - SW360 Package Creator - - [NPM/NUGET/MAVEN/PYTHON](../usagedocimg/packageCreatirnpmnuget.PNG) + - [NPM/NUGET/MAVEN/PYTHON/CONAN](../usagedocimg/packageCreatirnpmnuget.PNG) - [Debian](../usagedocimg/packagecreatordebian.PNG) - Artifactory Uploader - - [NPM/NUGET/MAVEN/PYTHON](../usagedocimg/artifactoryuploader.PNG) + - [NPM/NUGET/MAVEN/PYTHON/CONAN](../usagedocimg/artifactoryuploader.PNG) # Prerequisite @@ -158,11 +159,18 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and mvn clean install -DskipTests=true - - **Project Type :** **Python** + - **Project Type :** **Python** * Input file repository should contain **poetry.lock** file. - - - **Project Type :** **Debian** + + + - **Project Type :** **Conan** + + * Input file repository should contain **conan.lock** file. + + `Note : Conan package support in clearing tool is currently only for SBOM discovery and classification.Component Creation and Source code identification is not supported currently` + + - **Project Type :** **Debian** **Note** : below steps is required only if you have `tar` file to process , otherwise you can keep `CycloneDx.json` file in the InputDirectory. * Create `InputImage` directory for keeping `tar` images and `InputDirectory` for resulted file storing . @@ -260,11 +268,21 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and "Include": [ "poetry.lock", "*.cdx.json" ], "Exclude": [], "JfrogPythonRepoList": [ - , //This is a mirror repo for pypi in JFrog - "" //This should be the release pypi in JFrog + "", + "",//This should be the release repo in JFrog + ], + "ExcludedComponents": [] + }, + "Conan": { + "Include": [ "conan.lock"], + "Exclude": [], + "JfrogConanRepoList": [ + "", + "", ], "ExcludedComponents": [] } + } ``` diff --git a/src/LCT.APICommunications/ApiConstant.cs b/src/LCT.APICommunications/ApiConstant.cs index d004b5a8..c957e1b3 100644 --- a/src/LCT.APICommunications/ApiConstant.cs +++ b/src/LCT.APICommunications/ApiConstant.cs @@ -29,6 +29,7 @@ public static class ApiConstant public const string ComponentNameUrl = "?name="; public const string NPMExternalID = "pkg:npm/"; public const string NugetExternalID = "pkg:nuget/"; + public const string ConanExternalID = "pkg:conan/"; public const string NpmExtension = ".tgz"; public const string NugetExtension = ".nupkg"; public const string MavenExtension = "-sources.jar"; diff --git a/src/LCT.Common/CommonAppSettings.cs b/src/LCT.Common/CommonAppSettings.cs index ac9580e9..223ad72d 100644 --- a/src/LCT.Common/CommonAppSettings.cs +++ b/src/LCT.Common/CommonAppSettings.cs @@ -65,6 +65,7 @@ public CommonAppSettings(IFolderAction iFolderAction) public Config Maven { get; set; } public Config Debian { get; set; } public Config Python { get; set; } + public Config Conan { get; set; } public string CaVersion { get; set; } public string CycloneDxSBomTemplatePath { get; set; } public string[] InternalRepoList { get; set; } diff --git a/src/LCT.Common/CommonHelper.cs b/src/LCT.Common/CommonHelper.cs index 55162fe5..fb2d069e 100644 --- a/src/LCT.Common/CommonHelper.cs +++ b/src/LCT.Common/CommonHelper.cs @@ -5,6 +5,7 @@ // -------------------------------------------------------------------------------------------------------------------- using CycloneDX.Models; +using LCT.Common.Constants; using LCT.Common.Model; using log4net; using log4net.Core; @@ -228,5 +229,18 @@ public static bool ComponentPropertyCheck(Component component, string constant) } return component.Properties.Exists(x => x.Name == constant); } + + public static void GetDetailsforManuallyAdded(List componentsForBOM, List listComponentForBOM) + { + foreach (var component in componentsForBOM) + { + component.Properties = new List(); + Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = Dataconstant.ManullayAdded }; + component.Properties.Add(isDev); + component.Properties.Add(identifierType); + listComponentForBOM.Add(component); + } + } } } diff --git a/src/LCT.Common/Constants/Dataconstant.cs b/src/LCT.Common/Constants/Dataconstant.cs index 3f8d7504..0b30c82d 100644 --- a/src/LCT.Common/Constants/Dataconstant.cs +++ b/src/LCT.Common/Constants/Dataconstant.cs @@ -22,6 +22,7 @@ public static class Dataconstant {"DEBIAN", "pkg:deb/debian"}, {"MAVEN", "pkg:maven"}, {"PYTHON", "pkg:pypi"}, + {"CONAN", "pkg:conan"}, }; //Identified types diff --git a/src/LCT.Common/CycloneDXBomParser.cs b/src/LCT.Common/CycloneDXBomParser.cs index 7cb0bada..7c143bd4 100644 --- a/src/LCT.Common/CycloneDXBomParser.cs +++ b/src/LCT.Common/CycloneDXBomParser.cs @@ -72,8 +72,8 @@ public static Bom ExtractSBOMDetailsFromTemplate(Bom template) } //Taking SBOM Template Metadata - bom.Metadata = template?.Metadata; - bom.Dependencies = template?.Dependencies; + bom.Metadata = template.Metadata; + bom.Dependencies = template.Dependencies; return bom; } diff --git a/src/LCT.Common/Model/Config.cs b/src/LCT.Common/Model/Config.cs index 9b0f4ca3..6bc38c08 100644 --- a/src/LCT.Common/Model/Config.cs +++ b/src/LCT.Common/Model/Config.cs @@ -22,6 +22,7 @@ public class Config public string[] JfrogNugetRepoList { get; set; } public string[] JfrogMavenRepoList { get; set; } public string[] JfrogPythonRepoList { get; set; } + public string[] JfrogConanRepoList { get; set; } public string[] DevDependentScopeList { get; set; } } diff --git a/src/LCT.Common/appSettings.json b/src/LCT.Common/appSettings.json index 3089715e..8dc8a0cf 100644 --- a/src/LCT.Common/appSettings.json +++ b/src/LCT.Common/appSettings.json @@ -75,5 +75,14 @@ "" //This should be the release pypi in JFrog ], "ExcludedComponents": [] + }, + "Conan": { + "Include": [ "conan.lock" ], + "Exclude": [], + "JfrogConanRepoList": [ + "", //This is a mirror repo for conan in JFrog + "" //This should be the release repo in JFrog + ], + "ExcludedComponents": [] } } diff --git a/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs b/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs new file mode 100644 index 00000000..295755ef --- /dev/null +++ b/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs @@ -0,0 +1,247 @@ +// -------------------------------------------------------------------------------------------------------------------- +// SPDX-FileCopyrightText: 2023 Siemens AG +// +// SPDX-License-Identifier: MIT +// -------------------------------------------------------------------------------------------------------------------- + +using CycloneDX.Models; +using LCT.APICommunications.Model.AQL; +using LCT.Common; +using LCT.Common.Model; +using LCT.PackageIdentifier; +using LCT.PackageIdentifier.Interface; +using LCT.PackageIdentifier.Model; +using LCT.Services.Interface; +using Moq; +using NUnit.Framework; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace PackageIdentifier.UTest +{ + [TestFixture] + public class ConanParserTests + { + [TestCase] + public void ParseLockFile_GivenAInputFilePath_ReturnsSuccess() + { + //Arrange + int expectedNoOfcomponents = 17; + string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location; + string outFolder = Path.GetDirectoryName(exePath); + string packagefilepath = outFolder + @"\PackageIdentifierUTTestFiles"; + + string[] Includes = { "conan.lock" }; + Config config = new Config() + { + Include = Includes + }; + + CommonAppSettings appSettings = new CommonAppSettings() + { + PackageFilePath = packagefilepath, + Conan = config + }; + + //Act + Bom listofcomponents = new ConanProcessor().ParsePackageFile(appSettings); + + //Assert + Assert.That(expectedNoOfcomponents, Is.EqualTo(listofcomponents.Components.Count), "Checks for no of components"); + + } + + [TestCase] + public void ParseLockFile_GivenAInputFilePath_ReturnDevDependentComp() + { + //Arrange + string IsDev = "true"; + string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location; + string outFolder = Path.GetDirectoryName(exePath); + string packagefilepath = outFolder + @"\PackageIdentifierUTTestFiles"; + + string[] Includes = { "conan.lock" }; + Config config = new Config() + { + Include = Includes + }; + + CommonAppSettings appSettings = new CommonAppSettings() + { + PackageFilePath = packagefilepath, + Conan = config + }; + + //Act + Bom listofcomponents = new ConanProcessor().ParsePackageFile(appSettings); + var IsDevDependency = listofcomponents.Components.Find(a => a.Name == "googletest") + .Properties.First(x => x.Name == "internal:siemens:clearing:development").Value; + + //Assert + Assert.That(IsDev, Is.EqualTo(IsDevDependency), "Checks if Dev Dependency Component or not"); + + } + + [TestCase] + public void ParseLockFile_GivenAInputFilePathExcludeComponent_ReturnComponentCount() + { + //Arrange + int totalComponentsAfterExclusion = 15; + string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location; + string outFolder = Path.GetDirectoryName(exePath); + string packagefilepath = outFolder + @"\PackageIdentifierUTTestFiles"; + + string[] Includes = { "conan.lock" }; + Config config = new Config() + { + Include = Includes, + ExcludedComponents = new List { "openldap:2.6.4-shared-ossl3.1", "libcurl:7.87.0-shared-ossl3.1" } + }; + + CommonAppSettings appSettings = new CommonAppSettings() + { + PackageFilePath = packagefilepath, + Conan = config + }; + + //Act + Bom listofcomponents = new ConanProcessor().ParsePackageFile(appSettings); + + //Assert + Assert.That(totalComponentsAfterExclusion, Is.EqualTo(listofcomponents.Components.Count), "Checks if the excluded components have been removed"); + } + + [TestCase] + public void IsDevDependent_GivenListOfDevComponents_ReturnsSuccess() + { + //Arrange + var conanPackage = new ConanPackage() {Id = "10"}; + var buildNodeIds = new List { "10", "11", "12" }; + var noOfDevDependent = 0; + //Act + bool actual = ConanProcessor.IsDevDependency(conanPackage, buildNodeIds, ref noOfDevDependent); + + //Assert + Assert.That(true, Is.EqualTo(actual), "Component is a dev dependent"); + } + + [Test] + public async Task IdentificationOfInternalComponents_ReturnsComponentData_Successfully() + { + // Arrange + Component component = new Component() + { + Name = "securitycommunicationmanager", + Description = string.Empty, + Version = "2.6.5", + Purl = "pkg:conan/securitycommunicationmanager@2.6.5" + }; + + var components = new List() { component }; + ComponentIdentification componentIdentification = new() { comparisonBOMData = components }; + string[] repoList = { "internalrepo1", "internalrepo2" }; + CommonAppSettings appSettings = new() { InternalRepoList = repoList }; + + AqlResult aqlResult = new() + { + Name = "index.json", + Path = "siemens-energy/securitycommunicationmanager/2.7.1/stable", + Repo = "internalrepo1" + }; + + List results = new List() { aqlResult }; + Mock mockJfrogService = new Mock(); + Mock mockBomHelper = new Mock(); + mockBomHelper.Setup(m => m.GetListOfComponentsFromRepo(It.IsAny(), It.IsAny())) + .ReturnsAsync(results); + + // Act + ConanProcessor conanProcessor = new ConanProcessor(); + var actual = await conanProcessor.IdentificationOfInternalComponents(componentIdentification, appSettings, mockJfrogService.Object, mockBomHelper.Object); + + // Assert + Assert.That(actual, Is.Not.Null); + } + + [Test] + public async Task GetJfrogRepoDetailsOfAComponent_ReturnsWithData_SuccessFully() + { + // Arrange + Component component = new Component() + { + Name = "securitycommunicationmanager", + Description = string.Empty, + Version = "2.6.5", + Purl = "pkg:conan/securitycommunicationmanager@2.6.5" + }; + var components = new List() { component }; + string[] repoList = { "internalrepo1", "internalrepo2" }; + CommonAppSettings appSettings = new(); + appSettings.Conan = new LCT.Common.Model.Config() { JfrogConanRepoList = repoList }; + AqlResult aqlResult = new() + { + Name = "index.json", + Path = "siemens-energy/securitycommunicationmanager/2.6.5/stable", + Repo = "internalrepo1" + }; + + List results = new List() { aqlResult }; + + Mock mockJfrogService = new Mock(); + Mock mockBomHelper = new Mock(); + mockBomHelper.Setup(m => m.GetListOfComponentsFromRepo(It.IsAny(), It.IsAny())) + .ReturnsAsync(results); + + // Act + ConanProcessor conanProcessor = new ConanProcessor(); + var actual = await conanProcessor.GetJfrogRepoDetailsOfAComponent( + components, appSettings, mockJfrogService.Object, mockBomHelper.Object); + var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-url").Properties[0].Value; + + // Assert + Assert.That(actual, Is.Not.Null); + Assert.That(aqlResult.Repo, Is.EqualTo(reponameActual)); + } + + [Test] + public async Task GetArtifactoryRepoName_Conan_ReturnsNotFound_ReturnsFailure() + { + // Arrange + Component component = new Component() + { + Name = "securitycommunicationmanager", + Description = string.Empty, + Version = "2.6.5", + Purl = "pkg:conan/securitycommunicationmanager@2.6.5" + }; + var components = new List() { component }; + string[] repoList = { "internalrepo1", "internalrepo2" }; + CommonAppSettings appSettings = new(); + appSettings.Conan = new LCT.Common.Model.Config() { JfrogConanRepoList = repoList }; + AqlResult aqlResult = new() + { + Name = "index.json", + Path = "siemens-energy/securitycommunicationmanager/2.7.1/stable", + Repo = "internalrepo1" + }; + + List results = new() { aqlResult }; + + Mock mockJfrogService = new Mock(); + Mock mockBomHelper = new Mock(); + mockBomHelper.Setup(m => m.GetListOfComponentsFromRepo(It.IsAny(), It.IsAny())) + .ReturnsAsync(results); + + // Act + ConanProcessor conanProcessor = new ConanProcessor(); + var actual = await conanProcessor.GetJfrogRepoDetailsOfAComponent( + components, appSettings, mockJfrogService.Object, mockBomHelper.Object); + + var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-url").Properties[0].Value; + + Assert.That("Not Found in JFrogRepo", Is.EqualTo(reponameActual)); + } + } +} diff --git a/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj b/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj index 651788d3..e2a08c85 100644 --- a/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj +++ b/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj @@ -55,6 +55,9 @@ + + Always + Always diff --git a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/conan.lock b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/conan.lock new file mode 100644 index 00000000..2a45747b --- /dev/null +++ b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/conan.lock @@ -0,0 +1,154 @@ +{ + "graph_lock": { + "nodes": { + "0": { + "options": "libcurl:static=None\nopenssl:static=False", + "requires": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16" + ], + "build_requires": [ + "17" + ], + "path": "conan\\conanfile\\linux-x86_64", + "context": "host" + }, + "1": { + "ref": "rapidjson/1.1.0-csc-01@siemens-energy/stable#6d624490b731387491675eebeff7ab66", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "85ca839500f017c06cd5c39b4ad50c5e", + "context": "host" + }, + "2": { + "ref": "libest/3.2.0-shared.5@siemens-energy/stable#0", + "options": "", + "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", + "prev": "0", + "context": "host" + }, + "3": { + "ref": "openssl/3.0.9-shared.3@siemens-energy/stable#0", + "options": "static=False", + "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", + "prev": "0", + "context": "host" + }, + "4": { + "ref": "sqlite/3.37.0-shared.1@siemens-energy/stable#0", + "options": "", + "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", + "prev": "0", + "context": "host" + }, + "5": { + "ref": "oss_mbedtls/2.28.2-shared@siemens-energy/stable#0", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "0", + "context": "host" + }, + "6": { + "ref": "mongoose/v7.11-csc-01@siemens-energy/stable#7084912b9de8ac5f93c2e8aa16c259a8", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "c4b1dedbd2bd5223337ce892732c693e", + "context": "host" + }, + "7": { + "ref": "basics/1.0.8@siemens-energy/stable#0", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "0", + "context": "host" + }, + "8": { + "ref": "osal/1.0.30@siemens-energy/stable#21e224648b08c0c356e2e60d0f143ae7", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "f5db57253fc37159485da5d9d37d4844", + "context": "host" + }, + "9": { + "ref": "SecurityBasics/2.10.2@siemens-energy/stable#0", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "0", + "context": "host" + }, + "10": { + "ref": "securityaccessmanager/2.2.6@siemens-energy/stable#7522ada1783555e22afd338d9bd03d60", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "493721829e0ba6c4cccb9304acc9ad71", + "context": "host" + }, + "11": { + "ref": "SecurityEventLogger/2.0.24@siemens-energy/stable#ddd584e3d5551e3ba568f07b95a9e6af", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "f32d263a0ba627319365f9ee41781589", + "context": "host" + }, + "12": { + "ref": "securitypkimanager/2.6.3@siemens-energy/stable#a8d80c7af932e2062513e94fd2001077", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "33612b4203a7c283c1dd56aa795863f6", + "context": "host" + }, + "13": { + "ref": "SecurityStorageManager/2.11.2@siemens-energy/stable#6c98465724cb00ab842f16a6a17c64b6", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "8f87b12afc830478491b566d2b7bf7e1", + "context": "host" + }, + "14": { + "ref": "securitycommunicationmanager/2.6.5@siemens-energy/stable#fbacb77f419f7c1dc2af769841266fba", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "dcd1c2048a844e785cb79e918452d56e", + "context": "host" + }, + "15": { + "ref": "openldap/2.6.4-shared-ossl3.1@siemens-energy/stable#0", + "options": "", + "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", + "prev": "0", + "context": "host" + }, + "16": { + "ref": "libcurl/7.87.0-shared-ossl3.1@siemens-energy/stable#0", + "options": "static=None", + "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", + "prev": "0", + "context": "host" + }, + "17": { + "ref": "googletest/1.8.0@siemens-energy/stable#0", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "0", + "context": "host" + } + }, + "revisions_enabled": true + }, + "version": "0.4", + "profile_host": "[settings]\narch=x86_64\narch_build=x86_64\nbuild_type=Release\ncompiler=gcc\ncompiler.libcxx=libstdc++11\ncompiler.version=9.3\nos=Linux\nos_build=Linux\nswsign:os=Windows\nswsign:os_build=Windows\nswsign:compiler=Visual Studio\nswsign:compiler.version=15\nswsign:build_type=Release\nswsign:compiler.runtime=MD\nmbedtls_csc:os=Windows\nmbedtls_csc:os_build=Windows\nmbedtls_csc:compiler=Visual Studio\nmbedtls_csc:compiler.version=15\nmbedtls_csc:build_type=Release\nmbedtls_csc:compiler.runtime=MD\n[options]\nopenssl:static=False\n[build_requires]\n[env]\n" +} \ No newline at end of file diff --git a/src/LCT.PackageIdentifier/BomCreator.cs b/src/LCT.PackageIdentifier/BomCreator.cs index ea26b94b..14e083f7 100644 --- a/src/LCT.PackageIdentifier/BomCreator.cs +++ b/src/LCT.PackageIdentifier/BomCreator.cs @@ -119,6 +119,9 @@ private async Task CallPackageParser(CommonAppSettings appSettings) case "PYTHON": parser = new PythonProcessor(); return await ComponentIdentification(appSettings, parser); + case "CONAN": + parser = new ConanProcessor(); + return await ComponentIdentification(appSettings, parser); default: Logger.Error($"GenerateBom():Invalid ProjectType - {appSettings.ProjectType}"); break; diff --git a/src/LCT.PackageIdentifier/ConanProcessor.cs b/src/LCT.PackageIdentifier/ConanProcessor.cs new file mode 100644 index 00000000..04065bfd --- /dev/null +++ b/src/LCT.PackageIdentifier/ConanProcessor.cs @@ -0,0 +1,453 @@ +// -------------------------------------------------------------------------------------------------------------------- +// SPDX-FileCopyrightText: 2023 Siemens AG +// +// SPDX-License-Identifier: MIT +// -------------------------------------------------------------------------------------------------------------------- + +using CycloneDX.Models; +using LCT.APICommunications; +using LCT.APICommunications.Model.AQL; +using LCT.Common; +using LCT.Common.Constants; +using LCT.PackageIdentifier.Interface; +using LCT.PackageIdentifier.Model; +using LCT.Services.Interface; +using log4net; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Security; +using System.Threading.Tasks; + +namespace LCT.PackageIdentifier +{ + + /// + /// Parses the Conan Packages + /// + public class ConanProcessor : CycloneDXBomParser, IParser + { + #region fields + static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + readonly CycloneDXBomParser cycloneDXBomParser; + private const string NotFoundInRepo = "Not Found in JFrogRepo"; + #endregion + + #region constructor + public ConanProcessor() + { + if (cycloneDXBomParser == null) + { + cycloneDXBomParser = new CycloneDXBomParser(); + } + } + #endregion + + #region public methods + public Bom ParsePackageFile(CommonAppSettings appSettings) + { + List componentsForBOM; + Bom bom = new Bom(); + + ParsingInputFileForBOM(appSettings, ref bom); + componentsForBOM = bom.Components; + + componentsForBOM = GetExcludedComponentsList(componentsForBOM); + componentsForBOM = componentsForBOM.Distinct(new ComponentEqualityComparer()).ToList(); + + var componentsWithMultipleVersions = componentsForBOM.GroupBy(s => s.Name) + .Where(g => g.Count() > 1).SelectMany(g => g).ToList(); + + if (componentsWithMultipleVersions.Count != 0) + { + Logger.Warn($"Multiple versions detected :\n"); + foreach (var item in componentsWithMultipleVersions) + { + Logger.Warn($"Component Name : {item.Name}\nComponent Version : {item.Version}\nPackage Found in : {item.Description}\n"); + } + } + + bom.Components = componentsForBOM; + Logger.Debug($"ParsePackageFile():End"); + return bom; + } + + public async Task IdentificationOfInternalComponents(ComponentIdentification componentData, CommonAppSettings appSettings, + IJFrogService jFrogService, IBomHelper bomhelper) + { + // get the component list from Jfrog for given repository + List aqlResultList = + await bomhelper.GetListOfComponentsFromRepo(appSettings.InternalRepoList, jFrogService); + + // find the components in the list of internal components + List internalComponents = new List(); + var internalComponentStatusUpdatedList = new List(); + var inputIterationList = componentData.comparisonBOMData; + + foreach (Component component in inputIterationList) + { + var currentIterationItem = component; + bool isTrue = IsInternalConanComponent(aqlResultList, currentIterationItem); + if (currentIterationItem.Properties?.Count == null || currentIterationItem.Properties?.Count <= 0) + { + currentIterationItem.Properties = new List(); + } + + Property isInternal = new() { Name = Dataconstant.Cdx_IsInternal, Value = "false" }; + if (isTrue) + { + internalComponents.Add(currentIterationItem); + isInternal.Value = "true"; + } + else + { + isInternal.Value = "false"; + } + + currentIterationItem.Properties.Add(isInternal); + internalComponentStatusUpdatedList.Add(currentIterationItem); + } + + // update the comparison BOM data + componentData.comparisonBOMData = internalComponentStatusUpdatedList; + componentData.internalComponents = internalComponents; + + return componentData; + } + + + public async Task> GetJfrogRepoDetailsOfAComponent(List componentsForBOM, + CommonAppSettings appSettings, IJFrogService jFrogService, IBomHelper bomhelper) + { + // get the component list from Jfrog for given repository + List aqlResultList = await bomhelper.GetListOfComponentsFromRepo(appSettings.Conan?.JfrogConanRepoList, jFrogService); + Property projectType = new() { Name = Dataconstant.Cdx_ProjectType, Value = appSettings.ProjectType }; + List modifiedBOM = new List(); + + foreach (var component in componentsForBOM) + { + string repoName = GetArtifactoryRepoName(aqlResultList, component); + Property artifactoryrepo = new() { Name = Dataconstant.Cdx_ArtifactoryRepoUrl, Value = repoName }; + Component componentVal = component; + + if (componentVal.Properties?.Count == null || componentVal.Properties?.Count <= 0) + { + componentVal.Properties = new List(); + } + componentVal.Properties.Add(artifactoryrepo); + componentVal.Properties.Add(projectType); + componentVal.Description = string.Empty; + + modifiedBOM.Add(componentVal); + } + + return modifiedBOM; + } + + public static bool IsDevDependency(ConanPackage component, List buildNodeIds, ref int noOfDevDependent) + { + var isDev = false; + if (buildNodeIds != null && buildNodeIds.Contains(component.Id)) + { + isDev = true; + noOfDevDependent++; + } + + return isDev; + } + + #endregion + + #region private methods + private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref Bom bom) + { + List configFiles; + List dependencies = new List(); + List componentsForBOM = new List(); + configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Conan); + + foreach (string filepath in configFiles) + { + if (filepath.ToLower().EndsWith("conan.lock")) + { + Logger.Debug($"ParsingInputFileForBOM():FileName: " + filepath); + var components = ParsePackageLockJson(filepath, ref dependencies); + AddingIdentifierType(components, "PackageFile"); + componentsForBOM.AddRange(components); + } + else if (filepath.EndsWith(FileConstant.CycloneDXFileExtension) && !filepath.EndsWith(FileConstant.SBOMTemplateFileExtension)) + { + Logger.Debug($"ParsingInputFileForBOM():Found as CycloneDXFile"); + bom = cycloneDXBomParser.ParseCycloneDXBom(filepath); + CheckValidComponentsForProjectType(bom.Components, appSettings.ProjectType); + componentsForBOM.AddRange(bom.Components); + GetDetailsforManuallyAddedComp(componentsForBOM); + } + } + + int initialCount = componentsForBOM.Count; + GetDistinctComponentList(ref componentsForBOM); + BomCreator.bomKpiData.DuplicateComponents = initialCount - componentsForBOM.Count; + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = componentsForBOM.Count; + bom.Components = componentsForBOM; + + if (bom.Dependencies != null) + { + bom.Dependencies.AddRange(dependencies); + } + else + { + bom.Dependencies = dependencies; + } + + if (File.Exists(appSettings.CycloneDxSBomTemplatePath) && appSettings.CycloneDxSBomTemplatePath.EndsWith(FileConstant.SBOMTemplateFileExtension)) + { + //Adding Template Component Details + Bom templateDetails; + templateDetails = ExtractSBOMDetailsFromTemplate(cycloneDXBomParser.ParseCycloneDXBom(appSettings.CycloneDxSBomTemplatePath)); + CheckValidComponentsForProjectType(templateDetails.Components, appSettings.ProjectType); + SbomTemplate.AddComponentDetails(bom.Components, templateDetails); + } + + bom = RemoveExcludedComponents(appSettings, bom); + } + + private static List ParsePackageLockJson(string filepath, ref List dependencies) + { + List lstComponentForBOM = new List(); + int noOfDevDependent = 0; + + try + { + string jsonContent = File.ReadAllText(filepath); + var jsonDeserialized = JObject.Parse(jsonContent); + var nodes = jsonDeserialized["graph_lock"]["nodes"]; + + List nodePackages = new List(); + foreach (var node in nodes) + { + string nodeId = ((JProperty)node).Name; + var conanPackage = JsonConvert.DeserializeObject(((JProperty)node).Value.ToString()); + conanPackage.Id = nodeId; + nodePackages.Add(conanPackage); + } + + GetPackagesForBom(ref lstComponentForBOM, ref noOfDevDependent, nodePackages); + + GetDependecyDetails(lstComponentForBOM, nodePackages, dependencies); + + BomCreator.bomKpiData.DevDependentComponents += noOfDevDependent; + } + catch (JsonReaderException ex) + { + Environment.ExitCode = -1; + Logger.Error($"ParsePackageFile():", ex); + } + catch (IOException ex) + { + Environment.ExitCode = -1; + Logger.Error($"ParsePackageFile():", ex); + } + catch (SecurityException ex) + { + Environment.ExitCode = -1; + Logger.Error($"ParsePackageFile():", ex); + } + + return lstComponentForBOM; + } + + private static void GetDependecyDetails(List componentsForBOM, List nodePackages, List dependencies) + { + foreach (Component component in componentsForBOM) + { + var node = nodePackages.Find(x => x.Reference.Contains($"{component.Name}/{component.Version}")); + var dependencyNodes = new List(); + if (node.Dependencies != null && node.Dependencies.Count > 0) + { + dependencyNodes.AddRange(nodePackages.Where(x => node.Dependencies.Contains(x.Id)).ToList()); + } + if (node.DevDependencies != null && node.DevDependencies.Count > 0) + { + dependencyNodes.AddRange(nodePackages.Where(x => node.DevDependencies.Contains(x.Id)).ToList()); + } + var dependency = new Dependency(); + var subDependencies = componentsForBOM.Where(x => dependencyNodes.Exists(y => y.Reference.Contains($"{x.Name}/{x.Version}"))) + .Select(x => new Dependency { Ref = x.Purl }).ToList(); + + dependency.Ref = component.Purl; + dependency.Dependencies = subDependencies; + + if (subDependencies.Count > 0) + { + dependencies.Add(dependency); + } + } + } + + private static void GetPackagesForBom(ref List lstComponentForBOM, ref int noOfDevDependent, List nodePackages) + { + var rootNode = nodePackages.FirstOrDefault(); + if (!rootNode.Dependencies.Any() || rootNode.Dependencies == null) + { + throw new ArgumentNullException(nameof(nodePackages), "Dependency(requires) node name details not present in the root node."); + } + + // Ignoring the root node as it is the package information node and we are anyways considering all + // nodes in the lock file. + foreach (var component in nodePackages.Skip(1)) + { + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile += 1; + Property isdev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + + if (string.IsNullOrEmpty(component.Reference)) + { + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile--; + continue; + } + + Component components = new Component(); + + // dev components are not ignored and added as a part of SBOM + var buildNodeIds = GetBuildNodeIds(nodePackages); + if (IsDevDependency(component, buildNodeIds, ref noOfDevDependent)) + { + isdev.Value = "true"; + } + + string packageName = Convert.ToString(component.Reference); + + if (packageName.Contains('/')) + { + components.Name = packageName.Split(new char[] { '/', '@' })[0]; + components.Version = packageName.Split(new char[] { '/', '@' })[1]; + } + else + { + components.Name = packageName; + } + + components.Purl = $"{ApiConstant.ConanExternalID}{components.Name}@{components.Version}"; + components.BomRef = $"{ApiConstant.ConanExternalID}{components.Name}@{components.Version}"; + components.Properties = new List(); + components.Properties.Add(isdev); + lstComponentForBOM.Add(components); + } + } + + private static List GetBuildNodeIds(List nodePackages) + { + return nodePackages + .Where(y => y.DevDependencies != null) + .SelectMany(y => y.DevDependencies) + .ToList(); + } + + private static bool IsInternalConanComponent(List aqlResultList, Component component) + { + string jfrogcomponentPath = $"{component.Name}/{component.Version}"; + if (aqlResultList.Exists( + x => x.Path.Contains(jfrogcomponentPath, StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + return false; + } + + private static string GetArtifactoryRepoName(List aqlResultList, Component component) + { + string jfrogcomponentPath = $"{component.Name}/{component.Version}"; + + string repoName = aqlResultList.Find(x => x.Path.Contains( + jfrogcomponentPath, StringComparison.OrdinalIgnoreCase))?.Repo ?? NotFoundInRepo; + + return repoName; + } + + private static List GetExcludedComponentsList(List componentsForBOM) + { + List components = new List(); + foreach (Component componentsInfo in componentsForBOM) + { + if (!string.IsNullOrEmpty(componentsInfo.Name) && !string.IsNullOrEmpty(componentsInfo.Version) && !string.IsNullOrEmpty(componentsInfo.Purl) && componentsInfo.Purl.Contains(Dataconstant.PurlCheck()["CONAN"])) + { + components.Add(componentsInfo); + Logger.Debug($"GetExcludedComponentsList():ValidComponent For CONAN : Component Details : {componentsInfo.Name} @ {componentsInfo.Version} @ {componentsInfo.Purl}"); + } + else + { + BomCreator.bomKpiData.ComponentsExcluded++; + Logger.Debug($"GetExcludedComponentsList():InvalidComponent For CONAN : Component Details : {componentsInfo.Name} @ {componentsInfo.Version} @ {componentsInfo.Purl}"); + } + } + return components; + } + + private static void AddingIdentifierType(List components, string identifiedBy) + { + foreach (var component in components) + { + if (component.Properties == null) + { + component.Properties = new List(); + } + + Property isDev; + Property identifierType; + if (identifiedBy == "PackageFile") + { + identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = Dataconstant.Discovered }; + component.Properties.Add(identifierType); + } + else + { + isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = Dataconstant.ManullayAdded }; + component.Properties.Add(isDev); + component.Properties.Add(identifierType); + } + } + } + + private static void GetDistinctComponentList(ref List listofComponents) + { + int initialCount = listofComponents.Count; + listofComponents = listofComponents.GroupBy(x => new { x.Name, x.Version, x.Purl }).Select(y => y.First()).ToList(); + + if (listofComponents.Count != initialCount) + BomCreator.bomKpiData.DuplicateComponents = initialCount - listofComponents.Count; + } + + private static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) + { + List componentForBOM = cycloneDXBOM.Components.ToList(); + int noOfExcludedComponents = 0; + if (appSettings.Conan.ExcludedComponents != null) + { + componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Conan.ExcludedComponents, ref noOfExcludedComponents); + BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; + } + cycloneDXBOM.Components = componentForBOM; + return cycloneDXBOM; + } + + private static void GetDetailsforManuallyAddedComp(List componentsForBOM) + { + foreach (var component in componentsForBOM) + { + component.Properties = new List(); + Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = Dataconstant.ManullayAdded }; + component.Properties.Add(isDev); + component.Properties.Add(identifierType); + } + } + + #endregion + } +} \ No newline at end of file diff --git a/src/LCT.PackageIdentifier/Model/ConanPackage.cs b/src/LCT.PackageIdentifier/Model/ConanPackage.cs new file mode 100644 index 00000000..1498dfb7 --- /dev/null +++ b/src/LCT.PackageIdentifier/Model/ConanPackage.cs @@ -0,0 +1,26 @@ + +// -------------------------------------------------------------------------------------------------------------------- +// SPDX-FileCopyrightText: 2023 Siemens AG +// +// SPDX-License-Identifier: MIT + +// -------------------------------------------------------------------------------------------------------------------- + +using Newtonsoft.Json; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace LCT.PackageIdentifier.Model +{ + [ExcludeFromCodeCoverage] + public class ConanPackage + { + public string Id { get; set; } + [JsonProperty("ref")] + public string Reference { get; set; } = string.Empty; + [JsonProperty("requires")] + public List Dependencies { get; set; } + [JsonProperty("build_requires")] + public List DevDependencies { get; set; } + } +} diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index 3a800b48..2221b406 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -381,7 +381,7 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List componentsForBOM, List listComponentForBOM) - { - foreach (var component in componentsForBOM) - { - component.Properties = new List(); - Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; - Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = Dataconstant.ManullayAdded }; - component.Properties.Add(isDev); - component.Properties.Add(identifierType); - listComponentForBOM.Add(component); - } - } - private static void ConvertToCycloneDXModel(List listComponentForBOM, List listofComponents, List dependencies) { foreach (var prop in listofComponents) diff --git a/src/LCT.PackageIdentifier/PythonProcessor.cs b/src/LCT.PackageIdentifier/PythonProcessor.cs index cd0f7b66..85500a85 100644 --- a/src/LCT.PackageIdentifier/PythonProcessor.cs +++ b/src/LCT.PackageIdentifier/PythonProcessor.cs @@ -50,7 +50,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) foreach (string config in configFiles) { - if (config.EndsWith("poetry.lock")) + if (config.ToLower().EndsWith("poetry.lock")) { listofComponents.AddRange(ExtractDetailsForPoetryLockfile(config, dependencies)); } diff --git a/src/LCT.Services/Sw360Service.cs b/src/LCT.Services/Sw360Service.cs index 22a448e3..7bb77a9d 100644 --- a/src/LCT.Services/Sw360Service.cs +++ b/src/LCT.Services/Sw360Service.cs @@ -67,7 +67,7 @@ public async Task> GetAvailableReleasesInSw360(List if (modelMappedObject != null) { - availableComponentsList = await GetAvailableComponenentsList(modelMappedObject?.Embedded?.Sw360Releases, listOfComponentsToBom); + availableComponentsList = await GetAvailableComponenentsList(modelMappedObject.Embedded?.Sw360Releases, listOfComponentsToBom); } } catch (HttpRequestException ex)